import pandas as pd
import numpy as np

#importing our visualisation libraries
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

#removing the limit for the number of displayed columns
pd.set_option("display.max_columns", None)
#setting a maximum display of 2 decimal places for pandas
pd.options.display.float_format = "{:,.2f}".format

#importing models for regression
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

#importing our decision tree classifier models
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from xgboost import XGBClassifier

#importing the library to encode categorical variables
from sklearn.preprocessing import LabelEncoder

#importing GridSearchCV for hyperparameter tuning
from sklearn.model_selection import GridSearchCV

#importing metrics to check model performance
from sklearn.metrics import confusion_matrix, recall_score, precision_score, accuracy_score, classification_report, make_scorer

#importing the warnings library and muting warnings
import warnings
warnings.filterwarnings("ignore")

df = pd.read_csv('hmeq.csv')

df.head()

df.tail()

df.shape

(5960, 13)

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5960 entries, 0 to 5959
Data columns (total 13 columns):
 #   Column   Non-Null Count  Dtype  
---  ------   --------------  -----  
 0   BAD      5960 non-null   int64  
 1   LOAN     5960 non-null   int64  
 2   MORTDUE  5442 non-null   float64
 3   VALUE    5848 non-null   float64
 4   REASON   5708 non-null   object 
 5   JOB      5681 non-null   object 
 6   YOJ      5445 non-null   float64
 7   DEROG    5252 non-null   float64
 8   DELINQ   5380 non-null   float64
 9   CLAGE    5652 non-null   float64
 10  NINQ     5450 non-null   float64
 11  CLNO     5738 non-null   float64
 12  DEBTINC  4693 non-null   float64
dtypes: float64(9), int64(2), object(2)
memory usage: 605.4+ KB

df.duplicated().value_counts()

False    5960
Name: count, dtype: int64

df.nunique()

BAD           2
LOAN        540
MORTDUE    5053
VALUE      5381
REASON        2
JOB           6
YOJ          99
DEROG        11
DELINQ       14
CLAGE      5314
NINQ         16
CLNO         62
DEBTINC    4693
dtype: int64

df.isnull().sum()

BAD           0
LOAN          0
MORTDUE     518
VALUE       112
REASON      252
JOB         279
YOJ         515
DEROG       708
DELINQ      580
CLAGE       308
NINQ        510
CLNO        222
DEBTINC    1267
dtype: int64

df.describe().T

df.describe(include = 'object').T

#Defining our function to plot a combined plot of histrogram, boxplot and violin plot for numeric univariate analysis.. 
#We could choose to customise the parameters every time we call the function: however, we will
#only call it a limited number of times through this EDA as we will use a for loop to 
#iterate through the variables when calling it.

#This is the primary function we will use for our univariate analysis of numeric features.

def histogram_boxplot_violin(data, variable):
    #Here we create a subplot and allocate it to different plot types that we are creating, making them share
    #an x-axis for every variable, and allocating our common figsize
    figure, (ax_box, ax_histogram, ax_violin) = plt.subplots(3, sharex=True, figsize=(10, 8))
    #here we create our histogram, with kde=true to give us a better visualisation of distribution of the variable
    sns.histplot(data=data, x=variable, kde=True, ax = ax_histogram)
    #now we create our boxplot, and having it show our mean on the plot directly.
    sns.boxplot(data=data, x=variable, ax=ax_box, showmeans=True, color = 'lightblue')
    #and, lasttly, we create our violin plot, and have it display the quartile split to show us the distribution by
    #quartile for each variable intuitively
    sns.violinplot(data=data, x=variable, ax=ax_violin, inner='quartile', color = 'green')
    
    #we create vertical lines of consistent colors to show the median and means on each plot
    ax_histogram.axvline(data[variable].mean(), color = 'red', linestyle = '-')
    ax_histogram.axvline(data[variable].median(), color = 'purple', linestyle = '-')
    ax_violin.axvline(data[variable].mean(), color = 'red', linestyle = '-')
    ax_violin.axvline(data[variable].median(), color = 'purple', linestyle = '-')
    
    #Now we assign each plot a title. We use suptitle as opposed to title so it doesn't show individual titles for every
    #subplot
    plt.suptitle(variable.upper())
    #and here we have it show us plot without a warning message
    plt.show()

#defining our barplot piechart combined graph to perform univariate analysis on categorical features.

def barplot_piechart(data, variable, figsize=(10, 7)):
    #assigning subplots to our bar plots and pie charts. Also, this time I am making the figsize modifiable as the
    #counts and proportions within each plot may vary widely, so it will be more convenient for me to find the best
    #visualisation size this way
    figure, (ax_bar, ax_pie) = plt.subplots(2, figsize=figsize)
    #creating our bar plot
    sns.countplot(data=data, x=variable, ax=ax_bar)
    #creating our pie chart, including the labels for it
    count = data[variable].value_counts()
    label = count.index
    size = count.values
    #when creating a pie chart, we assign it directly to an ax rather than giving it as an argument.
    #additionally, we have it display both the labels and the percentages for each value in every feature.
    ax_pie.pie(size, labels=label, autopct='%1.1f%%', colors=['gold','red','lightgreen'])
    #the reason we assign 3 colours only is because there is a max of 3 variables that are seen in any given feature.
    
    plt.suptitle(variable.upper())
    plt.show()

#defining our numeric and categorical columns
num_cols = df.select_dtypes(include=['number']).columns.to_list()
cat_cols = df.select_dtypes(include=['object']).columns.to_list()

for col in df[num_cols]:
    histogram_boxplot_violin(df, col)

df.describe().T

cat_cols.append('BAD')

for col in cat_cols:
    barplot_piechart(df, col)
    plt.show()

#looking at the distribution of loan values sorted by whether the customer defaulted on a loan or not
sns.boxplot(df, x='BAD', y='LOAN')
plt.show()

sns.boxplot(df, x='BAD', y='LOAN', hue='REASON')
plt.show()

sns.countplot(df, x='JOB', hue='BAD')
plt.show()

sns.countplot(df, x='REASON', hue='BAD')
plt.show()

sns.histplot(df, x='MORTDUE', hue='BAD', alpha=0.7, kde=True)
plt.show()

corr_columns = df[(df.select_dtypes(include=[np.number]).columns.tolist())].corr()
corr_columns

mask = np.triu(corr_columns)
#now, we create our heatmap based on the correlations above, with the mask applied, annotating it, choosing a classic
#blue-red colour scheme, and with the max value being 1. Since we do not have any correlations below -0.1, we will not
#define a lower bound via vmin. Additionally, whilst we do include vmax = 1 here, it is more for sanity than
#a precise use as we know from the above table that none of the correlations go above 0.31.
sns.heatmap(corr_columns, mask=mask, annot = True, cmap = 'coolwarm', vmax = 1, fmt='.2f')
plt.show()

client_profile_3d = px.scatter_3d(df, x='YOJ', y='LOAN', z='BAD', color='JOB',
                                  labels = dict(YOJ='Years in current role', LOAN='Loan Value', 
                                               BAD='Loan Defaulted or Repaid'))
client_profile_3d.show()

property_financials_3d = px.scatter_3d(df, x='LOAN', y='VALUE', z='MORTDUE', color='BAD',
                                  labels = dict(VALUE='Total Property Value', LOAN='Loan Value', 
                                               BAD='Loan Defaulted or Repaid', MORTDUE='Mortgage Due'))
property_financials_3d.show()

financial_history_3d = px.scatter_3d(df, x='CLNO', y='DELINQ', z='DEROG', color='JOB',
                                  labels = dict(CLNO='Total No. Of Credit Lines', JOB='Current Job', 
                                               DELINQ='No. of Delinquent Credit Lines', DEROG='No. of Derogatory Reports'))
financial_history_3d.show()

client_overall_profile = px.scatter(df, x='VALUE', y='DELINQ', color='JOB',
                                   labels={'VALUE': 'Total Property Value',
                                          'DELINQ': 'No. of Delinquent Credit Lines'})
client_overall_profile.show()

client_overall_profile2 = px.scatter(df, x='MORTDUE', y='DEROG', color='JOB',
                                   labels={'MORTDUE': 'Total Mortgage Due',
                                          'DEROG': 'No. of Derogatory Reports'})
client_overall_profile2.show()

managers_df = df[df['JOB'] == 'Mgr']
managers_df['BAD'].value_counts()

BAD
0    588
1    179
Name: count, dtype: int64

df.isnull().sum()

BAD           0
LOAN          0
MORTDUE     518
VALUE       112
REASON      252
JOB         279
YOJ         515
DEROG       708
DELINQ      580
CLAGE       308
NINQ        510
CLNO        222
DEBTINC    1267
dtype: int64

null_rows = df[df.isnull().any(axis=1)]
null_rows

df['REASON'].value_counts()

REASON
DebtCon    3928
HomeImp    1780
Name: count, dtype: int64

df['JOB'].value_counts()

JOB
Other      2388
ProfExe    1276
Office      948
Mgr         767
Self        193
Sales       109
Name: count, dtype: int64

df['REASON'].isnull().sum()

252

df['JOB'].isnull().sum()

279

missing_cat = df[df['REASON'].isnull() & df['JOB'].isnull()]
missing_cat.shape

(107, 13)

missing_cat_index = df[df['REASON'].isnull() & df['JOB'].isnull()].index
missing_cat_index

Index([   3,   10,   17,   51,   73,  112,  115,  143,  237,  268,
       ...
       4418, 4581, 4632, 4660, 4680, 4880, 4947, 5331, 5348, 5468],
      dtype='int64', length=107)

df = df.drop(missing_cat_index, axis = 0)

df[df['REASON'].isnull() & df['JOB'].isnull()]

df['REASON'].fillna((df['REASON'].mode()[0]), inplace=True)
df['JOB'].fillna((df['JOB'].mode()[0]), inplace=True)

df.isnull().sum()

BAD           0
LOAN          0
MORTDUE     445
VALUE       100
REASON        0
JOB           0
YOJ         432
DEROG       623
DELINQ      495
CLAGE       210
NINQ        425
CLNO        137
DEBTINC    1249
dtype: int64

df.shape

(5853, 13)

df.describe().T

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.neighbors import KNeighborsRegressor

#initialising our KNN regressor with 10 neighbours. This will serve as our estimator.
knn_regressor = KNeighborsRegressor(n_neighbors=10)
#creating our iterative imputer. In addition to our default parameters, we specify the 'ascending' 
#imputation order to ensure that the data with the least missing features is filled first, to give us
#more accurate imputation in rows with a significant number of missing features.
imputer = IterativeImputer(estimator=knn_regressor,  
                           imputation_order='ascending', 
                           random_state=0)

#defining our numeric and categorical columns, so we can have the imputer fill in only missing values
#in numeric columns, as we have already treated our categorical columns.
num_cols = df.select_dtypes(include=[np.number]).columns
cat_cols = df.select_dtypes(exclude=[np.number]).columns

df_num_imputed = pd.DataFrame(imputer.fit_transform(df[num_cols]), columns=num_cols)

df = pd.concat([df[cat_cols].reset_index(drop=True), df_num_imputed.reset_index(drop=True)], axis=1)

df

#creating a function for our log transformation of the data.
def log_transformation_treatment(data, feature):
    #taking the log of our feature (and adding 1 to handle 0 values)
    data[feature] = np.log(data[feature] + 1)
    
    #defining our quantiles and interquartile range
    q1 = np.percentile(data[feature], 25)
    q3 = np.percentile(data[feature], 75)
    iqr = q3 - q1
    #defining our upper and lower bounds
    lower = q1 - (iqr * 1.5)
    upper = q3 + (iqr * 1.5)
    
    #capping our features at both the upper or lower quantile
    data[feature] = np.clip(data[feature], lower, upper)
    
    #taking the exponent of our data in order to return it to the original scale
    data[feature] = np.exp(data[feature]) - 1
    
    return data

#Removing our target variable, as well as the features where both Q1 and Q3 are 0 as seen in EDA
num_cols_list = num_cols.tolist()
num_cols_list.remove('BAD')
num_cols_list.remove('DEROG')
num_cols_list.remove('DELINQ')
num_cols = pd.Index(num_cols_list)

#applying our log transformation to lessen the impact of outliers in our models.
for feature in df[num_cols]:
    log_transformation_treatment(df, feature)

df.describe().T

#making our categorical columns index into a list to make the next step easy
cat_cols = cat_cols.tolist()

#one-hot encoding our categoric variables
df = pd.get_dummies(df, columns = cat_cols, drop_first=True, dtype='int')
df

df.drop('VALUE', axis=1, inplace=True)
df.head()

#Separating our target and non-target variables
x = df.drop('BAD', axis=1)

y = df['BAD']

#Separating our data into train and test sets
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, shuffle=True, random_state = 0)

# Checking the shape of the train and test data
print("Shape of Training set : ", x_train.shape, y_train.shape)
print("Shape of test set : ", x_test.shape, y_test.shape)

Shape of Training set :  (4682, 15) (4682,)
Shape of test set :  (1171, 15) (1171,)

#Adapted from: HR Employee Attrition Prediction notebook by Great Learning.

# Creating metric function

def metrics_score(actual, predicted):

    print(classification_report(actual, predicted))

    cm = confusion_matrix(actual, predicted)

    plt.figure(figsize = (8, 5))

    sns.heatmap(cm, annot = True, fmt = '.2f', xticklabels = ['Not Defaulted', 'Defaulted'], yticklabels = ['Not Defaulted', 'Defaulted'])
    plt.ylabel('Actual')

    plt.xlabel('Predicted')

    plt.show()
    

# Creating the Classification Performance Metrics table

def model_performance_classification(model, predictors, target):
    """
    Function to compute different metrics to check classification model performance

    model: classifier

    predictors: independent variables

    target: dependent variable
    """

    # Predicting using the independent variables, as well as setting our positive label
    
    pos_label = 1
    
    pred = model.predict(predictors)

    recall = recall_score(target, pred, pos_label=pos_label, average = 'macro')                 # To compute recall

    precision = precision_score(target, pred, pos_label=pos_label, average = 'macro')              # To compute precision

    acc = accuracy_score(target, pred)                                 # To compute accuracy score
    
    recall_class_1 = recall_score(target, pred, pos_label=1, average = 'binary')            #To compute recall score for our defaulting clients


    # Creating a dataframe of metrics

    df_perf = pd.DataFrame(
        {
            "Precision":  precision,
            "Recall":  recall,
            "Recall (Defaulting Loans)": recall_class_1,
            "Accuracy": acc,
        },

        index = [0],
    )

    return df_perf

# Creating the Linear Regression for Binary Classification Performance Metrics table

def model_performance_regression(model, predictors, target, threshold):
    """
    Function to compute different metrics to check linear regression for binary classification model performance

    model: lineear regression

    predictors: independent variables

    target: dependent variable
    
    threshold: decision threshold for binary classification
    """
    
    # Setting our positive label
    
    pos_label = 1
    
    # Predicting Probabilities of the target variable
    pred_prob = model.predict_proba(predictors)[:, 1]
    
    # Setting our threshold
    
    pred = (pred_prob >= threshold).astype(int)
    
    

    recall = recall_score(target, pred, pos_label=pos_label, average = 'macro')                 # To compute recall

    precision = precision_score(target, pred, pos_label=pos_label, average = 'macro')              # To compute precision

    acc = accuracy_score(target, pred)                                 # To compute accuracy score
    
    recall_class_1 = recall_score(target, pred, pos_label=1, average = 'binary')            #To compute recall score for our defaulting clients


    # Creating a dataframe of metrics

    df_perf = pd.DataFrame(
        {
            "Precision":  precision,
            "Recall":  recall,
            "Recall (Defaulting Loans)": recall_class_1,
            "Accuracy": acc,
        },

        index = [0],
    )

    return df_perf

from sklearn.preprocessing import StandardScaler

#initialising our scaler
scaler = StandardScaler()

#scaling the data
x_train_scaled = scaler.fit_transform(x_train)
x_test_scaled = scaler.transform(x_test)

#defining and fitting our model on our training data
logistic_reg = LogisticRegression()
logistic_reg.fit(x_train_scaled, y_train)

LogisticRegression()

LogisticRegression()

#predicting the probability of our hypothesis, and selecting the probability of BAD=1
#as our target
y_pred_prob = logistic_reg.predict_proba(x_test_scaled)[:, 1]

y_pred_prob

array([0.12298145, 0.16711751, 0.19980537, ..., 0.16059954, 0.10812457,
       0.04791157])

from sklearn.metrics import precision_recall_curve, auc

precision, recall, threshold = precision_recall_curve(y_test, y_pred_prob)

#Adapted from: 'Logistic Regression for Binary Classification Task' by Fares Sayah, Kaggle
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g--", label="Recall")
    plt.xlabel("Threshold")
    plt.legend(loc="upper left")
    plt.title("Precisions/recalls tradeoff")

plt.figure(figsize=(15, 10))
plot_precision_recall_vs_threshold(precision, recall, threshold)

y_pred_binary = (y_pred_prob >= 0.25).astype(int)

metrics_score(y_test, y_pred_binary)

lr_metrics = model_performance_regression(logistic_reg, x_test_scaled, y_test, 0.25)
lr_metrics

              precision    recall  f1-score   support

         0.0       0.87      0.86      0.87       910
         1.0       0.54      0.56      0.55       261

    accuracy                           0.80      1171
   macro avg       0.71      0.71      0.71      1171
weighted avg       0.80      0.80      0.80      1171

y_pred_binary1 = (y_pred_prob >= 0.11).astype(int)

metrics_score(y_test, y_pred_binary1)

              precision    recall  f1-score   support

         0.0       0.93      0.46      0.62       910
         1.0       0.32      0.88      0.47       261

    accuracy                           0.56      1171
   macro avg       0.63      0.67      0.55      1171
weighted avg       0.80      0.56      0.59      1171

lr_perf = model_performance_classification(logistic_reg, x_test_scaled, y_test)
lr_perf

lr_tuned = LogisticRegression()

param_grid = {
    'penalty': ['l1', 'l2', 'elasticnet', 'none'],
    'class_weight': [{1: 0.5, 0: 0.5}, {1: 0.35, 0: 0.65}, {1: 0.2, 0: 0.8}],
    'C': [0.5, 0.6, 0.7, 0.8],
    'solver': ['liblinear', 'newton-cg', 'saga']
}

scorer = make_scorer(recall_score, pos_label=1)

lr_tuned_cv = GridSearchCV(
    estimator = lr_tuned,
    param_grid = param_grid,
    cv = 10,
    scoring = scorer)

lr_tuned_cv = lr_tuned_cv.fit(x_train_scaled, y_train)

lr_tuned_estimator = lr_tuned_cv.best_estimator_

lr_tuned_estimator.fit(x_train_scaled, y_train)

LogisticRegression(C=0.5, class_weight={0: 0.5, 1: 0.5}, solver='liblinear')

LogisticRegression(C=0.5, class_weight={0: 0.5, 1: 0.5}, solver='liblinear')

y_pred_prob_tuned = lr_tuned_estimator.predict_proba(x_test_scaled)[:, 1]

precision, recall, threshold = precision_recall_curve(y_test, y_pred_prob_tuned)

plt.figure(figsize=(15, 10))
plot_precision_recall_vs_threshold(precision, recall, threshold)

y_pred_binary_tuned = (y_pred_prob >= 0.25).astype(int)
metrics_score(y_test, y_pred_binary_tuned)
lr_tuned_metrics = model_performance_regression(lr_tuned_estimator, x_test_scaled, y_test, 0.25)
lr_tuned_metrics

              precision    recall  f1-score   support

         0.0       0.87      0.86      0.87       910
         1.0       0.54      0.56      0.55       261

    accuracy                           0.80      1171
   macro avg       0.71      0.71      0.71      1171
weighted avg       0.80      0.80      0.80      1171

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis

lda = LinearDiscriminantAnalysis()
lda.fit(x_train_scaled, y_train)

LinearDiscriminantAnalysis()

LinearDiscriminantAnalysis()

y_pred_train_lda = lda.predict(x_train_scaled)

metrics_score(y_train, y_pred_train_lda)

              precision    recall  f1-score   support

         0.0       0.85      0.96      0.90      3764
         1.0       0.66      0.29      0.40       918

    accuracy                           0.83      4682
   macro avg       0.75      0.62      0.65      4682
weighted avg       0.81      0.83      0.80      4682

y_pred_test_lda = lda.predict_proba(x_test_scaled)[:, 1]

precision, recall, threshold = precision_recall_curve(y_test, y_pred_test_lda)

plt.figure(figsize=(15, 10))
plot_precision_recall_vs_threshold(precision, recall, threshold)

y_pred_binary_lda = (y_pred_test_lda >= 0.22).astype(int)

metrics_score(y_test, y_pred_binary_lda)

lda_metrics = model_performance_regression(lda, x_test_scaled, y_test, 0.22)
lda_metrics

              precision    recall  f1-score   support

         0.0       0.87      0.87      0.87       910
         1.0       0.55      0.54      0.54       261

    accuracy                           0.80      1171
   macro avg       0.71      0.71      0.71      1171
weighted avg       0.80      0.80      0.80      1171

qda = QuadraticDiscriminantAnalysis()

qda.fit(x_train_scaled, y_train)

y_pred_train_qda = qda.predict(x_train_scaled)

metrics_score(y_train, y_pred_train_qda)

              precision    recall  f1-score   support

         0.0       0.86      0.89      0.88      3764
         1.0       0.48      0.40      0.44       918

    accuracy                           0.80      4682
   macro avg       0.67      0.65      0.66      4682
weighted avg       0.79      0.80      0.79      4682

y_pred_test_qda = qda.predict_proba(x_test_scaled)[:, 1]

precision, recall, threshold = precision_recall_curve(y_test, y_pred_test_qda)

plt.figure(figsize=(15, 10))
plot_precision_recall_vs_threshold(precision, recall, threshold)

y_pred_binary_qda = (y_pred_test_qda >= 0.16).astype(int)

metrics_score(y_test, y_pred_binary_qda)
qda_metrics = model_performance_regression(qda, x_test_scaled, y_test, 0.16)
qda_metrics

              precision    recall  f1-score   support

         0.0       0.86      0.89      0.87       910
         1.0       0.56      0.48      0.52       261

    accuracy                           0.80      1171
   macro avg       0.71      0.69      0.70      1171
weighted avg       0.79      0.80      0.79      1171

#Initialising our Decision Tree Classifier
dt = DecisionTreeClassifier(class_weight='balanced', random_state = 0)
#fitting the training data
dt.fit(x_train, y_train)

DecisionTreeClassifier(class_weight='balanced', random_state=0)

DecisionTreeClassifier(class_weight='balanced', random_state=0)

#let's check the performance of our tree on the training data.
train_dt_pred = dt.predict(x_train)

metrics_score(y_train, train_dt_pred)
model_performance_classification(dt, x_train, y_train)

              precision    recall  f1-score   support

         0.0       1.00      1.00      1.00      3764
         1.0       1.00      1.00      1.00       918

    accuracy                           1.00      4682
   macro avg       1.00      1.00      1.00      4682
weighted avg       1.00      1.00      1.00      4682

test_dt_pred = dt.predict(x_test)

metrics_score(y_test, test_dt_pred)
dt_metrics = model_performance_classification(dt, x_test, y_test)
dt_metrics

              precision    recall  f1-score   support

         0.0       0.89      0.94      0.91       910
         1.0       0.74      0.60      0.66       261

    accuracy                           0.86      1171
   macro avg       0.81      0.77      0.79      1171
weighted avg       0.86      0.86      0.86      1171

#Adapted from: HR Employee Attrition Prediction notebook by Great Learning.

#creating a function to plot the importances of features in our decision tree

def feature_importance_plot(model):
    #assigning our importances to a variable
    importances = model.feature_importances_

    columns = x.columns
    #buildinf a df of feature importances, sotred from most to least important features
    importance_df = pd.DataFrame(importances, index = columns, columns = ['Importance']).sort_values(by = 'Importance', ascending = False)

    plt.figure(figsize = (13, 13))
    #plotting our df
    sns.barplot(x=importance_df.Importance,y=importance_df.index)

    #displaying our feature importance ratio on the right of each feature's bar for ease of comprehension
    for index, value in enumerate(importance_df['Importance']):
        plt.text(value, index, f'{value:.2f}', va='center')

    plt.title("Feature Importances in Decision Tree Model");

feature_importance_plot(dt)

from sklearn import tree
features = list(x.columns)

dt_classifier_visualised = DecisionTreeClassifier(class_weight = 'balanced', max_depth = 3, random_state = 0)

dt_classifier_visualised.fit(x_train, y_train)

plt.figure(figsize = (20, 20))
tree.plot_tree(dt_classifier_visualised, feature_names = features, filled = True, fontsize = 11, class_names = ['Not Defaulted', 'Defaulted'])
plt.show()

#building our decision tree for the purposes of optimising its hyperparameters.
dtree_estimator = DecisionTreeClassifier(class_weight='balanced', random_state = 1)

#assigning a grid of parameters to choose from.
parameters = {
    'max_depth': np.arange(2, 8),
    'criterion': ['gini', 'entropy'],
    'min_samples_leaf': [5, 10, 15, 20, 25]
}
#designing our scoring system for GridSearchCV
scorer = make_scorer(recall_score, pos_label=1)

#running the grid search. The reason we use 10-fold cross validation here is due to the dataset being relatively small.
#the reason we do not use stratified k-fold despite the class imbalance is because we account for the imbalance in
#our estimator.
gridCV = GridSearchCV(dtree_estimator, parameters, scoring = scorer, cv=10)

#fitting the grid search on the training dataset
gridCV = gridCV.fit(x_train, y_train)
#setting the classifier to look for the best combination of parameters
dtree_estimator = gridCV.best_estimator_

#fitting the estimator to the data
dtree_estimator.fit(x_train, y_train)

DecisionTreeClassifier(class_weight='balanced', max_depth=7,
                       min_samples_leaf=10, random_state=1)

DecisionTreeClassifier(class_weight='balanced', max_depth=7,
                       min_samples_leaf=10, random_state=1)

#checking the performance of our optimised decision tree on the training data
y_train_dt_tuned_pred = dtree_estimator.predict(x_train)

metrics_score(y_train, y_train_dt_tuned_pred)

              precision    recall  f1-score   support

         0.0       0.94      0.78      0.85      3764
         1.0       0.47      0.81      0.60       918

    accuracy                           0.79      4682
   macro avg       0.71      0.80      0.73      4682
weighted avg       0.85      0.79      0.80      4682

#checking the performance of our optimised decision tree on the training data
y_test_dt_tuned_pred = dtree_estimator.predict(x_test)

metrics_score(y_test, y_test_dt_tuned_pred)

dt_tuned_metrics = model_performance_classification(dtree_estimator, x_test, y_test)
dt_tuned_metrics

              precision    recall  f1-score   support

         0.0       0.91      0.77      0.83       910
         1.0       0.47      0.73      0.57       261

    accuracy                           0.76      1171
   macro avg       0.69      0.75      0.70      1171
weighted avg       0.81      0.76      0.77      1171

feature_importance_plot(dtree_estimator)

#building our random forest classifier and fitting it with the training data
rf_classifier = RandomForestClassifier(class_weight = 'balanced', random_state=0)

rf_classifier.fit(x_train, y_train)

RandomForestClassifier(class_weight='balanced', random_state=0)

RandomForestClassifier(class_weight='balanced', random_state=0)

#checking the performance on the training data
y_pred_rf_train = rf_classifier.predict(x_train)

metrics_score(y_train, y_pred_rf_train)

              precision    recall  f1-score   support

         0.0       1.00      1.00      1.00      3764
         1.0       1.00      1.00      1.00       918

    accuracy                           1.00      4682
   macro avg       1.00      1.00      1.00      4682
weighted avg       1.00      1.00      1.00      4682

#checking the performance of the model on our training data
y_pred_rf_test = rf_classifier.predict(x_test)

metrics_score(y_test, y_pred_rf_test)
rf_metrics = model_performance_classification(rf_classifier, x_test, y_test)
rf_metrics

              precision    recall  f1-score   support

         0.0       0.90      1.00      0.95       910
         1.0       0.99      0.61      0.76       261

    accuracy                           0.91      1171
   macro avg       0.94      0.81      0.85      1171
weighted avg       0.92      0.91      0.90      1171

feature_importance_plot(rf_classifier)

#building our decision tree for the purposes of optimising its hyperparameters.
rf_classifier = RandomForestClassifier(class_weight='balanced', random_state = 0)

#assigning a grid of parameters to choose from.
parameters = {
    'max_depth': [5, 7, None], 
    'n_estimators': [100, 110, 120], 
    'max_features': [0.6, 0.8, 1]
}

#designing our scoring system for GridSearchCV
scorer = make_scorer(recall_score, pos_label=1)

#running the grid search. The reason we use 10-fold cross validation here is due to the dataset being relatively small.
#the reason we do not use stratified k-fold despite the class imbalance is because we account for the imbalance in
#our estimator.
gridCV = GridSearchCV(rf_classifier, parameters, scoring = scorer, cv=10)

#fitting the grid search on the training dataset
gridCV = gridCV.fit(x_train, y_train)
#setting the classifier to look for the best combination of parameters
rf_tuned = gridCV.best_estimator_

#fitting the estimator to the data
rf_tuned.fit(x_train, y_train)

RandomForestClassifier(class_weight='balanced', max_depth=5, max_features=1,
                       n_estimators=110, random_state=0)

RandomForestClassifier(class_weight='balanced', max_depth=5, max_features=1,
                       n_estimators=110, random_state=0)

y_pred_rf_tuned_train = rf_tuned.predict(x_train)
metrics_score(y_train, y_pred_rf_tuned_train)

              precision    recall  f1-score   support

         0.0       0.94      0.85      0.89      3764
         1.0       0.56      0.76      0.64       918

    accuracy                           0.84      4682
   macro avg       0.75      0.81      0.77      4682
weighted avg       0.86      0.84      0.84      4682

#checking the performance of the model on our training data
y_pred_rf_test = rf_tuned.predict(x_test)

metrics_score(y_test, y_pred_rf_test)

rf_tuned_metrics = model_performance_classification(rf_tuned, x_test, y_test)
rf_tuned_metrics

              precision    recall  f1-score   support

         0.0       0.91      0.83      0.87       910
         1.0       0.54      0.70      0.61       261

    accuracy                           0.80      1171
   macro avg       0.73      0.77      0.74      1171
weighted avg       0.83      0.80      0.81      1171

feature_importance_plot(rf_tuned)

gb_classifier = GradientBoostingClassifier(random_state = 1)

#fitting the classifier to our training data
gb_classifier.fit(x_train, y_train)

GradientBoostingClassifier(random_state=1)

GradientBoostingClassifier(random_state=1)

#checking the performance of the Gradient Boost model on our training data
y_pred_gb_classifier_train = gb_classifier.predict(x_train)
metrics_score(y_train, y_pred_gb_classifier_train)

              precision    recall  f1-score   support

         0.0       0.89      0.99      0.94      3764
         1.0       0.94      0.50      0.65       918

    accuracy                           0.90      4682
   macro avg       0.91      0.75      0.80      4682
weighted avg       0.90      0.90      0.88      4682

#checking the performance of the Gradient Boost model on our training data
y_pred_gb_classifier_test = gb_classifier.predict(x_test)
metrics_score(y_test, y_pred_gb_classifier_test)

gb_metrics = model_performance_classification(gb_classifier, x_test, y_test)
gb_metrics

              precision    recall  f1-score   support

         0.0       0.86      0.99      0.92       910
         1.0       0.90      0.46      0.61       261

    accuracy                           0.87      1171
   macro avg       0.88      0.72      0.77      1171
weighted avg       0.87      0.87      0.85      1171

feature_importance_plot(gb_classifier)

ada_classifier = AdaBoostClassifier(random_state = 1)

#fitting the classifier to our training data
ada_classifier.fit(x_train, y_train)

AdaBoostClassifier(random_state=1)

AdaBoostClassifier(random_state=1)

#checking the performance of the AdaBoost model on our training data
y_pred_ada_classifier_train = ada_classifier.predict(x_train)
metrics_score(y_train, y_pred_ada_classifier_train)

              precision    recall  f1-score   support

         0.0       0.88      0.97      0.92      3764
         1.0       0.76      0.44      0.56       918

    accuracy                           0.86      4682
   macro avg       0.82      0.70      0.74      4682
weighted avg       0.85      0.86      0.85      4682

#checking the performance of the AdaBoost model on our test data
y_pred_ada_classifier_test = ada_classifier.predict(x_test)
metrics_score(y_test, y_pred_ada_classifier_test)

ada_metrics = model_performance_classification(ada_classifier, x_test, y_test)
ada_metrics

              precision    recall  f1-score   support

         0.0       0.86      0.96      0.91       910
         1.0       0.78      0.44      0.56       261

    accuracy                           0.85      1171
   macro avg       0.82      0.70      0.73      1171
weighted avg       0.84      0.85      0.83      1171

feature_importance_plot(ada_classifier)

#building and fitting our XGB classifier to the training data. We will also precise the gpu_hist tree method in order
#to make use of the gpu to help accelerate the construction of our models. GPU accelerated prediction can additionally be
#disabled if need by setting predictor = 'cpu predictor' in order to conserve GPU memory.
xgb = XGBClassifier(device = 'cuda', tree_method = 'gpu_hist', class_weight='balanced', random_state=0)
xgb.fit(x_train, y_train)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              class_weight='balanced', colsample_bylevel=None,
              colsample_bynode=None, colsample_bytree=None, device='cuda',
              early_stopping_rounds=None, enable_categorical=False,
              eval_metric=None, feature_types=None, gamma=None,
              grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, ...)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              class_weight='balanced', colsample_bylevel=None,
              colsample_bynode=None, colsample_bytree=None, device='cuda',
              early_stopping_rounds=None, enable_categorical=False,
              eval_metric=None, feature_types=None, gamma=None,
              grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, ...)

y_pred_xgb_train = xgb.predict(x_train)

metrics_score(y_train, y_pred_xgb_train)

              precision    recall  f1-score   support

         0.0       1.00      1.00      1.00      3764
         1.0       1.00      0.99      1.00       918

    accuracy                           1.00      4682
   macro avg       1.00      1.00      1.00      4682
weighted avg       1.00      1.00      1.00      4682

y_pred_xgb_test = xgb.predict(x_test)
metrics_score(y_test, y_pred_xgb_test)

xgb_metrics = model_performance_classification(xgb, x_test, y_test)
xgb_metrics

              precision    recall  f1-score   support

         0.0       0.91      0.99      0.95       910
         1.0       0.93      0.66      0.77       261

    accuracy                           0.91      1171
   macro avg       0.92      0.82      0.86      1171
weighted avg       0.91      0.91      0.91      1171

feature_importance_plot(xgb)

#creating a df with our untuned model results

models_test_comp_df = pd.concat(
    [
        lr_metrics.T,
        lda_metrics.T,
        qda_metrics.T,
        dt_metrics.T,
        dt_tuned_metrics.T,
        rf_metrics.T,
        rf_tuned_metrics.T,
        gb_metrics.T,
        ada_metrics.T,
        xgb_metrics.T
    ],
    axis = 1,
)

models_test_comp_df.columns = [
    "Linear Regression Model",
    "Linear Discriminant Analysis Model",
    "Quadratic Discriminant Analysis Model",
    "Decision Tree Classifier",
    "Tuned Decision Tree Classifier",
    "Random Forest Classifier",
    "Tuned Random Forest Classifier",
    "Gradient Boost Classifier",
    "AdaBoost Classifier",
    "XGBoost Classifier"
    ]

print("Test performance comparison:")

models_test_comp_df.T

Test performance comparison:

from sklearn.model_selection import RandomizedSearchCV

#making a new weighted scorer, which prioritises recall on our defaulted loans, but also
#accounts for accuracy at a 9 to 1 ratio

def xgb_weighted_score(y_true, y_pred):
    recall = recall_score(y_true, y_pred, pos_label=1)
    accuracy = accuracy_score(y_true, y_pred)
    weighted_score = (3 * recall + 1 * accuracy) / 4
    return weighted_score

weighted_scorer = make_scorer(xgb_weighted_score)

xgb = XGBClassifier(device = 'gpu', random_state=0)

parameters_dist = {
    'max_depth': [3, 5, 7, 10],
    'learning_rate': [0.01, 0.1, 0.2, 0.3],
    'n_estimators': [50, 100, 150],
    'alpha': [0, 0.1, 1, 2],
    'lambda': [0, 0.1, 1, 2],
    'colsample_bytree': [0.3, 0.5, 0.7, 1],
    'subsample': [0.6, 0.8, 1],
    'gamma': [0, 0.1, 0.5, 1],
    'min_child_weight': [1, 3, 5],
    'scale_pos_weight': [1, 3, 5]
}

rand_search = RandomizedSearchCV(estimator=xgb, 
                                 param_distributions=parameters_dist, 
                                 n_iter=100, 
                                 scoring=weighted_scorer, 
                                 cv=10, 
                                 n_jobs=-1, 
                                 random_state=0)

rand_search.fit(x_train, y_train)

xgb_tuned = rand_search.best_estimator_

xgb_tuned.fit(x_train, y_train)

XGBClassifier(alpha=0, base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.3, device='gpu', early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=0.1, grow_policy=None, importance_type=None,
              interaction_constraints=None, lambda=2, learning_rate=0.01,
              max_bin=None, max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=10, max_leaves=None,
              min_child_weight=3, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=100, n_jobs=None, ...)

XGBClassifier(alpha=0, base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.3, device='gpu', early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=0.1, grow_policy=None, importance_type=None,
              interaction_constraints=None, lambda=2, learning_rate=0.01,
              max_bin=None, max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=10, max_leaves=None,
              min_child_weight=3, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=100, n_jobs=None, ...)

#Checking the performance of the model on the training data.
y_pred_xgb_tuned_train = xgb_tuned.predict(x_train)

metrics_score(y_train, y_pred_xgb_tuned_train)

              precision    recall  f1-score   support

         0.0       1.00      0.87      0.93      3764
         1.0       0.65      0.99      0.78       918

    accuracy                           0.89      4682
   macro avg       0.82      0.93      0.86      4682
weighted avg       0.93      0.89      0.90      4682

#Checking the performance of the model on the test data.
y_pred_xgb_tuned_test = xgb_tuned.predict(x_test)

metrics_score(y_test, y_pred_xgb_tuned_test)

xgb_tuned_metrics = model_performance_classification(xgb_tuned, x_test, y_test)
xgb_tuned_metrics

              precision    recall  f1-score   support

         0.0       0.95      0.79      0.87       910
         1.0       0.54      0.86      0.67       261

    accuracy                           0.81      1171
   macro avg       0.75      0.83      0.77      1171
weighted avg       0.86      0.81      0.82      1171

feature_importance_plot(xgb_tuned)

#pip install graphviz
import os
from xgboost import plot_tree
from graphviz import Source

# Function created with the aid of ChatGPT

# Function to visualize a tree with graphviz, allowing control over the figure size
def plot_tree_with_graphviz(booster, num_tree=0, max_depth=3):
    # Get the tree dump in DOT format
    tree_dump = booster.get_dump(dump_format='dot', with_stats=True)[num_tree]
    
    # Insert graph attributes to control size and direction
    tree_dump = 'digraph {\nsize="20,20";\nrankdir=UT;\n' + tree_dump[tree_dump.find('\n')+1:]
    
    # Create a Source object from the DOT format tree
    tree_graph = Source(tree_dump)
    
    # Render the tree graph to view it
    tree_graph.view()

# Example usage with xgb_tuned
num_trees = xgb_tuned.get_booster().num_boosted_rounds()
plot_tree_with_graphviz(xgb_tuned.get_booster(), num_tree=num_trees-1, max_depth=3)

#Adapted from Batfan's answer on stackoverflow: 
#https://stackoverflow.com/questions/291813/recommended-way-to-embed-pdf-in-html

from IPython.display import HTML

file_path = 'Source.gv.pdf'

# Display PDF with HTML
HTML(f'<embed src="{file_path}" width="800" height="600" type="application/pdf">')

	BAD	LOAN	MORTDUE	VALUE	REASON	JOB	YOJ	DEROG	DELINQ	CLAGE	NINQ	CLNO	DEBTINC
0	1	1100	25,860.00	39,025.00	HomeImp	Other	10.50	0.00	0.00	94.37	1.00	9.00	NaN
1	1	1300	70,053.00	68,400.00	HomeImp	Other	7.00	0.00	2.00	121.83	0.00	14.00	NaN
2	1	1500	13,500.00	16,700.00	HomeImp	Other	4.00	0.00	0.00	149.47	1.00	10.00	NaN
3	1	1500	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4	0	1700	97,800.00	112,000.00	HomeImp	Office	3.00	0.00	0.00	93.33	0.00	14.00	NaN

	LOAN	MORTDUE	VALUE	REASON	JOB	YOJ	CLAGE	CLNO	DEBTINC
5955	88900	57,264.00	90,185.00	DebtCon	Other	16.00	221.81	16.00	36.11
5956	89000	54,576.00	92,937.00	DebtCon	Other	16.00	208.69	15.00	35.86
5957	89200	54,045.00	92,924.00	DebtCon	Other	15.00	212.28	15.00	35.56
5958	89800	50,370.00	91,861.00	DebtCon	Other	14.00	213.89	16.00	34.34
5959	89900	48,811.00	88,934.00	DebtCon	Other	15.00	219.60	16.00	34.57

	count	mean	std	min	25%	50%	75%	max
BAD	5,960.00	0.20	0.40	0.00	0.00	0.00	0.00	1.00
LOAN	5,960.00	18,607.97	11,207.48	1,100.00	11,100.00	16,300.00	23,300.00	89,900.00
MORTDUE	5,442.00	73,760.82	44,457.61	2,063.00	46,276.00	65,019.00	91,488.00	399,550.00
VALUE	5,848.00	101,776.05	57,385.78	8,000.00	66,075.50	89,235.50	119,824.25	855,909.00
YOJ	5,445.00	8.92	7.57	0.00	3.00	7.00	13.00	41.00
DEROG	5,252.00	0.25	0.85	0.00	0.00	0.00	0.00	10.00
DELINQ	5,380.00	0.45	1.13	0.00	0.00	0.00	0.00	15.00
CLAGE	5,652.00	179.77	85.81	0.00	115.12	173.47	231.56	1,168.23
NINQ	5,450.00	1.19	1.73	0.00	0.00	1.00	2.00	17.00
CLNO	5,738.00	21.30	10.14	0.00	15.00	20.00	26.00	71.00
DEBTINC	4,693.00	33.78	8.60	0.52	29.14	34.82	39.00	203.31

	count	mean	std	min	25%	50%	75%	max
BAD	5,960.00	0.20	0.40	0.00	0.00	0.00	0.00	1.00
LOAN	5,960.00	18,607.97	11,207.48	1,100.00	11,100.00	16,300.00	23,300.00	89,900.00
MORTDUE	5,442.00	73,760.82	44,457.61	2,063.00	46,276.00	65,019.00	91,488.00	399,550.00
VALUE	5,848.00	101,776.05	57,385.78	8,000.00	66,075.50	89,235.50	119,824.25	855,909.00
YOJ	5,445.00	8.92	7.57	0.00	3.00	7.00	13.00	41.00
DEROG	5,252.00	0.25	0.85	0.00	0.00	0.00	0.00	10.00
DELINQ	5,380.00	0.45	1.13	0.00	0.00	0.00	0.00	15.00
CLAGE	5,652.00	179.77	85.81	0.00	115.12	173.47	231.56	1,168.23
NINQ	5,450.00	1.19	1.73	0.00	0.00	1.00	2.00	17.00
CLNO	5,738.00	21.30	10.14	0.00	15.00	20.00	26.00	71.00
DEBTINC	4,693.00	33.78	8.60	0.52	29.14	34.82	39.00	203.31

	BAD	LOAN	MORTDUE	VALUE	YOJ	DEROG	DELINQ	CLAGE	NINQ	CLNO	DEBTINC
BAD	1.00	-0.08	-0.05	-0.03	-0.06	0.28	0.35	-0.17	0.17	-0.00	0.20
LOAN	-0.08	1.00	0.23	0.34	0.11	-0.00	-0.04	0.09	0.04	0.07	0.08
MORTDUE	-0.05	0.23	1.00	0.88	-0.09	-0.05	-0.00	0.14	0.03	0.32	0.15
VALUE	-0.03	0.34	0.88	1.00	0.01	-0.05	-0.01	0.17	-0.00	0.27	0.13
YOJ	-0.06	0.11	-0.09	0.01	1.00	-0.07	0.04	0.20	-0.07	0.02	-0.06
DEROG	0.28	-0.00	-0.05	-0.05	-0.07	1.00	0.21	-0.08	0.17	0.06	0.02
DELINQ	0.35	-0.04	-0.00	-0.01	0.04	0.21	1.00	0.02	0.07	0.16	0.05
CLAGE	-0.17	0.09	0.14	0.17	0.20	-0.08	0.02	1.00	-0.12	0.24	-0.05
NINQ	0.17	0.04	0.03	-0.00	-0.07	0.17	0.07	-0.12	1.00	0.09	0.14
CLNO	-0.00	0.07	0.32	0.27	0.02	0.06	0.16	0.24	0.09	1.00	0.19
DEBTINC	0.20	0.08	0.15	0.13	-0.06	0.02	0.05	-0.05	0.14	0.19	1.00

Loan Default Prediction¶

Problem Definition¶

The Context:¶

The objective:¶

The key questions:¶

The problem formulation:¶

Data Description:¶

Import the necessary libraries and Data¶

Data Overview¶

Summary Statistics¶

Exploratory Data Analysis (EDA) and Visualization¶

Univariate Analysis¶

Bivariate Analysis¶

Multivariate Analysis¶

Treating Missing Values¶

Treating Outliers¶

Important Insights from EDA¶

Model Building - Approach¶

Data Preprocessing¶

References:¶

Logistic Regression¶

References¶

Decision Tree¶

Note on building the models¶

Decision Tree - Hyperparameter Tuning¶

Building a Random Forest Classifier¶

Random Forest Classifier Hyperparameter Tuning¶

Building a Gradient Boost model¶

Building an AdaBoost Classifier¶

Building an XGBoost Model¶

Model Optimisation Decision¶

XGBoost Hyperparameter Optimisation¶

References¶

Conclusion¶

Comparison of techniques, and their relative performance based on chosen Metric¶

References¶

	count	mean	std	min	25%	50%	75%	max
BAD	5,853.00	0.20	0.40	0.00	0.00	0.00	0.00	1.00
LOAN	5,853.00	18,696.40	11,253.38	1,100.00	11,100.00	16,400.00	23,400.00	89,900.00
MORTDUE	5,408.00	73,914.26	44,504.99	2,063.00	46,404.50	65,106.50	91,740.25	399,550.00
VALUE	5,753.00	102,112.99	57,638.65	8,000.00	66,193.00	89,673.00	120,000.00	855,909.00
YOJ	5,421.00	8.95	7.57	0.00	3.00	7.00	13.00	41.00
DEROG	5,230.00	0.25	0.84	0.00	0.00	0.00	0.00	10.00
DELINQ	5,358.00	0.45	1.13	0.00	0.00	0.00	0.00	15.00
CLAGE	5,643.00	179.83	85.85	0.00	115.17	173.49	231.67	1,168.23
NINQ	5,428.00	1.19	1.72	0.00	0.00	1.00	2.00	17.00
CLNO	5,716.00	21.35	10.10	0.00	15.00	20.00	26.00	71.00
DEBTINC	4,604.00	34.00	8.38	0.52	29.31	34.93	39.08	203.31

	BAD	LOAN	MORTDUE	VALUE	YOJ	DEROG	DELINQ	CLAGE	NINQ	CLNO	DEBTINC	REASON_HomeImp	JOB_Office	JOB_Other	JOB_ProfExe	JOB_Sales	JOB_Self
0	1.00	3,626.05	25,860.00	39,025.00	10.50	0.00	0.00	94.37	1.00	9.00	32.84	1	0	1	0	0	0
1	1.00	3,626.05	70,053.00	68,400.00	7.00	0.00	2.00	121.83	0.00	14.00	40.60	1	0	1	0	0	0
2	1.00	3,626.05	14,373.55	26,977.19	4.00	0.00	0.00	149.47	1.00	10.00	31.59	1	0	1	0	0	0
3	0.00	3,626.05	97,800.00	112,000.00	3.00	0.00	0.00	93.33	0.00	14.00	34.69	1	1	0	0	0	0
4	1.00	3,626.05	30,548.00	40,320.00	9.00	0.00	0.00	101.47	1.00	8.00	37.11	1	0	1	0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
5848	0.00	71,620.41	57,264.00	90,185.00	16.00	0.00	0.00	221.81	0.00	16.00	36.11	0	0	1	0	0	0
5849	0.00	71,620.41	54,576.00	92,937.00	16.00	0.00	0.00	208.69	0.00	15.00	35.86	0	0	1	0	0	0
5850	0.00	71,620.41	54,045.00	92,924.00	15.00	0.00	0.00	212.28	0.00	15.00	35.56	0	0	1	0	0	0
5851	0.00	71,620.41	50,370.00	91,861.00	14.00	0.00	0.00	213.89	0.00	16.00	34.34	0	0	1	0	0	0
5852	0.00	71,620.41	48,811.00	88,934.00	15.00	0.00	0.00	219.60	0.00	16.00	34.57	0	0	1	0	0	0

	Precision	Recall	Recall (Defaulting Loans)	Accuracy
Linear Regression Model	0.71	0.71	0.56	0.80
Linear Discriminant Analysis Model	0.71	0.71	0.54	0.80
Quadratic Discriminant Analysis Model	0.71	0.69	0.48	0.80
Decision Tree Classifier	0.81	0.77	0.60	0.86
Tuned Decision Tree Classifier	0.69	0.75	0.73	0.76
Random Forest Classifier	0.94	0.81	0.61	0.91
Tuned Random Forest Classifier	0.73	0.77	0.70	0.80
Gradient Boost Classifier	0.88	0.72	0.46	0.87
AdaBoost Classifier	0.82	0.70	0.44	0.85
XGBoost Classifier	0.92	0.82	0.66	0.91